
# 1. Select one Code (e.g. Corp)
# 2. Work backwards through statutes: does the statute change the selected Code
# 3. If it does, find all of the bill versions for the statute
# 4. Determine the original text for all of the sections changed by the statute
# 5. Save (a) the statute, (b) the original version, (c) the strikeout version, (d) the version info (effective date, etc.)
#from the list of statutes.sessions get list of libraries.identifier and of these, get list of identifier which end '-chp:library'

# Downloaded list of statute sessions from http://legix.info/us-ca/statutes/config.json; this is stored as statutes.json

# Dependencies:
# ujson or simplejson or json, urllib3, jsonpath, oset, BeautifulSoup

# jsonpath usage:
# jsonpath(obj, expr, result_type='VALUE', debug=0)

"""secversions.py - collects text of versions for a given section of code."""

import pickle
import re
from oset import oset as orderedset
try: import ujson as json
except ImportError:
    try: import simplejson as json
    except ImportError: import json
import urllib3
from jsonpath import jsonpath
from BeautifulSoup import BeautifulSoup

def defaultpool(hosturl="legix.info"):
    """ Create a default http pool connection to the hosturl """

    http_pool = urllib3.connection_from_url(hosturl, timeout=10.0)
    return http_pool

def getresponse(docurl, http_pool=defaultpool()):
    """ Gets an http response using a hostpool in urrlib3 """
    
    print "Getting response for " + docurl
    r = http_pool.get_url(docurl, retries=2)
    response = r.data
    return response

def legixresolve(slug, http_pool=defaultpool()):
    """Requests legix.info to resolve a urn and returns the result"""

    docurl = 'http://legix.info/resolve('+slug+')'
    print "Legixresolve for " + docurl
    response = getresponse(docurl, http_pool)
    return response

def jsonify(response):
    """Convert to unicode and returns a python structure from a json object"""

    print "Loading to json:" + response
    json_object = json.loads(response)
    return json_object

def getchapterids(chapteridsfile):
    with open(chapteridsfile, 'r') as chpids:
        chapteridslist = pickle.load(chpids)
    return chapteridslist

def getstatutesbychapterid(chapterid, http_pool=defaultpool()):
    slug = chapterid+':json'
    response = legixresolve(slug, http_pool)
    json_object = jsonify(response)
    statutesbychapterid = jsonpath(json_object,'documents..identifier')
    return statutesbychapterid 

def getcodesectionversions(json_object, secnum):
    sections = jsonpath(json_object, 'fragment.content.sections')[0]
    versionlist= [section for section in sections if jsonpath(section,'num')[0]==str(secnum)][0]['versions']
    srcvars = (version['src'] for version in versionlist)
    return srcvars

def getbillversions(srcvar, http_pool=defaultpool()):
    docurl = 'http://legix.info/us-ca/'+srcvar+'/config.json'
    json_object = jsonify(getresponse(docurl, http_pool))
    versionslist = jsonpath(json_object, 'versions')
    return versionslist 

#def getoriginaltextfromsrc(srcReflist, secnum, code): # e.g. secnum= 33050, code = "Education Code"
#    #Try versions from earliest to latest
#    for srcRef in reversed(srcReflist):
#        billtext = getresponse(srcRef)
#        # get the line with section + ' is ' +('amended'|'added'|'deleted') in it
#        sectextre = re.findall(r'Section\s'+str(secnum)+'\sof\sthe\s'+code+'\sis\s((?:amended)|(?:added)|(?:removed)).*?\:(.*?)(?:(?:SEC)|(?:<\/body>))',billvar, re.S)
#        if sectextre:
#            return sectextre
#    return None

#e.g. securn = u'urn:legix:us-ca:code-edc:doc(sec33050)'
# id in idlist = u'urn:legix:us-ca:1993_94-sb1537:doc-ver97'
def getoriginaltext(idlist, securn, http_pool=defaultpool()):
    """Find the first bill that changes a particular code section;
    Return the "fragment" metadata along with the original, redlined, text.
       """

    #Try versions from earliest to latest
    for identifier in reversed(idlist):
        billjson = jsonify(legixresolve(identifier+':json', http_pool))
        fragments = jsonpath(billjson,'sections.[?(@.fragment)]')
        print fragments
        print securn
        if fragments:
            print fragments[0]['fragment']['src']
            fragmenttarget = [fragment for fragment in fragments if fragment['fragment']['src']==securn]
            print fragmenttarget
            if fragmenttarget:
                print 'legixresolve '+ identifier +':'
                originalhtml = legixresolve(identifier+':html')
                #Grab the div where id=fragid
                soup = BeautifulSoup(originalhtml)
                fragid = fragmenttarget[0]['fragment']['content']['id']
                originaltext = soup.find(id = fragid)
                return fragmenttarget[0]['fragment'], originaltext
    return None

def storeversions(sectionhistory, securn):
    """ Save the dictionary from sectionhistory in a json file 
    with each fragment metadata followed by the redlined text, taken from the first bill
    in which a change to the section was introduced
       """
    filename = securn.replace(':','-') +'.txt'
    with open(filename, 'w') as shistfile:
        json.dump(sectionhistory, shistfile) # add'l options for simplejson: ensure_ascii=True, sort_keys=False, indent = 4

def main():
    # Constants
    chapteridslist = './chapters.pkl'
    hosturl = "http://legix.info"
    docurl = "http://legix.info/us-ca/code-prc/doc(div3-chp1-art4).json"
    secnum = 3201 
    securn = docurl.replace('http://legix.info','urn:legix').replace('/',':').rsplit(':',1)[0]+':doc(sec'+str(secnum)+')'
    http_pool = urllib3.connection_from_url(hosturl, timeout=60.0)
    spread = getresponse(docurl, http_pool)
    print 'got spread'
    spread = jsonify(spread)
    # TODO: loop over sections in the spread
    sectionhistory = []
    codesectionversions = getcodesectionversions(spread, secnum)
    print 'got section versions'
    #deduplicate code versions, remove any 'ver-98' variables
    codesrcvars = orderedset(x.split(':')[3].split('-ver')[0] for x in codesectionversions)
    #codesrcvars = orderedset(map(lambda x: x.split(':')[3].split('-ver')[0],codesectionversions))
    for version in codesrcvars:
        billversions = getbillversions(version, http_pool)
        print 'got bill versions for ' + version
        idlist = [x['identifier'] for x in billversions[0]]
        fragment, originaltext = getoriginaltext(idlist, securn, http_pool)
        originaltext = unicode(str(originaltext), errors="ignore")
        print originaltext
        sectionhistory.append({u'fragment':str(fragment), u'originaltext': str(originaltext)})
    storeversions(sectionhistory, securn)
    sectionhistory = json.dumps(sectionhistory) # add'l options for simplejson sort_keys=False, indent = 4
    return sectionhistory

# Get the chapters list and dump to a pickle
#def getchapterids():
#    with open('./statutes.json', 'r') as statutechapters:
#        chapters = jsonify(statutechapters.read())
#    chapteridslist = jsonpath(chapters, 'statutes.[*].sessions.[*].identifier')
#    #Remove resolutions
#    chapteridslist = [x for x in chapteridslist if x.split('-')[2][0:3]=='chp']
#    with open('./chapters.pkl', 'w') as chpids:
#        pickle.dump(chpids)
#    return chapteridslist

# urllib2 version
#def getjson(docurl):
#    req = urllib2.Request(docurl, "{'Content-Type': 'application/json'}")
#    f = urllib2.urlopen(req)
#    response = f.read()
#    f.close()
#    return response

#docurl = "http://legix.info/us-ca/code-edc/doc(ttl2-div2-prt20-chp1-art3).json"
#secnum = 33050

#docurl = "http://legix.info/us-ca/code-wat/doc(div7-chp3-art4).json"
#secnum = 13176
